import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;

public class LocalFile {
    public static void main(String[] args) {
        // 创建 SparkConf
        SparkConf conf = new SparkConf()
                .setAppName("LocalFile")
                .setMaster("local");

        // 创建 JavaDSparkContext
        JavaSparkContext sc = new JavaSparkContext(conf);

        // 使用 SparkContext 以及子类 读取本地文件
        JavaRDD<String> lines = sc.textFile("src/main/resources/spark.txt");

        // 统计文本文件内的字数
        JavaRDD<Integer> lineLength = lines.map(new Function<String, Integer>() {
            private static final long serialVersionUID = 1L;

            public Integer call(String s) throws Exception {
//                return s.length();  // 返回每一行的 字数
                return s.split(" ").length;  // 返回每一行的单词数
            }
        });

        int count = lineLength.reduce(new Function2<Integer, Integer, Integer>() {
            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });

        System.out.println("单词总数："+count);
        sc.close();
    }
}
